package com.asiainfo.zqx;

import cn.edu.hfut.dmic.webcollector.model.CrawlDatum;
import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;
import cn.edu.hfut.dmic.webcollector.model.Page;
import cn.edu.hfut.dmic.webcollector.plugin.berkeley.BreadthCrawler;
import cn.edu.hfut.dmic.webcollector.util.FileUtils;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.*;
import java.util.concurrent.atomic.AtomicInteger;

public class Taogezatan extends BreadthCrawler {
    private static final String seedurl = "https://c.m.163.com/news/sub/T1614331166577.html";
    private static final String Regurl = "https://c.m.163.com/news/a/.*";
    int num=0;
    String outFilePath;//爬取内容文件存储路径
    FileOutputStream out;
    // 用于保存图片的文件夹
    File downloadDir;

    // 原子性int，用于生成图片文件名
    AtomicInteger imageId;
    int num1=1;
    public Taogezatan(String crawlPath) throws IOException {
        super(crawlPath, false);
        CrawlDatum crawlDatum=new CrawlDatum(seedurl).meta("depth",3);
        addSeed(crawlDatum);
        addRegex(Regurl);
        outFilePath="./涛哥.txt";//爬取内容文件存储路径
        File file=new File(outFilePath);
        if(!file.exists())
            file.createNewFile();
        try {
            out=new FileOutputStream(file,true);
        } catch (FileNotFoundException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } //如果追加方式用true,默认false
        downloadDir = new File("taogeimage");
        if (!downloadDir.exists()) {
            downloadDir.mkdirs();
        }
        setThreads(3);
     //   setResumable(true);
    }

    @Override
    public void visit(Page page, CrawlDatums next) {

        //获取网页类型
        String contentType = page.contentType();
//
        if (contentType==null){
            return ;
        }

        if(page.matchUrl(Regurl)){
            System.out.println("开始判断");
            //获取标题
            String title=   page.select("div.header>h1.header-title").text();
            String title1="标题为:"+title+"\n";
            System.out.println("标题为"+title1);
            //获取时间
            String time=page.select("div.header-subtitle-middle>span.s-ptime").text();
            String time1=time+"\n";
            System.out.println("时间为"+time1);
            //time.substring(0,19)+
            //获取作者
            String writer=page.select("div.header-subtitle-middle>a").first().text();
            String writer1="作者为:"+writer+"\n";
            System.out.println("作者为"+writer1);
         /*   Elements elements=page.select("div.post_body>p");
            for(int i=0;i<elements.size();i++){
                Element element=elements.get(i);
                System.out.println(element.text());
            }*/
            //article.article-box-content n-platform-android>div>
            String text=page.select("p").text()+"\n"+"\n";
            System.out.println("正文为"+text);
            //Elements elements=page.select("div.post_body").select("img[src]");
            Elements elements=page.select("article.article-box-content n-platform-android>img[src]");
            if(elements.size()!=0){
                System.out.println("超链接个数"+elements.size());
                for(Element elemen:elements){


                    CrawlDatum crawlDatum=new CrawlDatum(elemen.attr("abs:src")).meta("refer",page.url()).meta("depth",1);
                    next.add(crawlDatum);
                }
            }

            //.select("img[src]");

            //将爬取的当前页内容写入文件
            try {
                out.write(title1.getBytes("utf-8"));
                out.write(time1.getBytes("utf-8"));
                out.write(writer1.getBytes("utf-8"));
                out.write(text.getBytes("utf-8"));
                //System.out.println(text);
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
        }
        else if(contentType.startsWith("image")){
            // 如果是图片，直接下载
            String extensionName = contentType.split("/")[1];
            System.out.println("后罪名"+extensionName);
            //String imageFileName = imageId.incrementAndGet() + "." + extensionName;
            String imageFileName = num1+ "." + extensionName;
            num1++;
            System.out.println("文件名"+imageFileName);
            File imageFile = new File(downloadDir, imageFileName);
            try {
                FileUtils.write(imageFile, page.content());
                System.out.println("保存图片 " + page.url() + " 到 " + imageFile.getAbsolutePath());
            } catch (IOException ex) {
                throw new RuntimeException(ex);
            }
        }
        else {
            Elements elements = page.select("a");
            for (int i = 0; i < elements.size(); i++) {
                Element element = elements.get(i);
                System.out.println("连接为"+element.attr("abs:href"));
                if (element.attr("abs:href").matches(Regurl)) {
                    System.out.println(element.attr("abs:href")+"连接成功加入");
                    CrawlDatum craw = new CrawlDatum(element.attr("abs:href")).meta("depth", 2)
                            .meta("refer", page.url());
                    next.add(craw);
                    num++;
                }
            }
        }
    }

    public static void main(String[] args) throws Exception {
        Taogezatan taoge= new Taogezatan("taoge");
        taoge.start(3);
        taoge.out.close();
        System.out.println("爬取页数"+taoge.num);
    }
}

